from __future__ import division
from operator import itemgetter, attrgetter
from struct import *
import gc
import math
import os
import random
import sys
import time
from sets import Set
from random import randint
import re
import numpy as np
#import matplotlib.pyplot as plt
#from statsmodels import api as sm

#101999 378757 1744742 2801091 8637794 133815.7346
print 101999 * math.log(378757/101999)
exit(1)

QPCInTotal = 0
queryLengthAndTotalResultsDict = {}
queryLengthAndTotalCostDict = {}
queryLengthAndNumOfQueriesBelongingToDict = {}

qidAndQPCForMaxScoreDict = {}
inputFileName = "/home/vgc/wei/workspace/NYU_IRTK/results/usefulInfoForMaxScore_20140715_attempt2"
inputFileHandler = open(inputFileName,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    QID = lineElements[0]
    totalQPC = lineElements[2]
    qidAndQPCForMaxScoreDict[QID] = totalQPC
print "len(qidAndQPCForMaxScoreDict):",len(qidAndQPCForMaxScoreDict)

termANDUnprunedListLengthDict = {}
# for gov2
# for vidaserver1:
inputFileName0 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/queryProcessingCostAnalysis/gov2/queryTermsWithTheirLengthsOfInvertedList"
# for clueweb09B
# inputFileName0 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/partOfLexiconTermsWithTermIDForTestingQueries_clueweb09B"
inputFileHandler = open(inputFileName0,"r")
currentLine = inputFileHandler.readline()
currentLineNum = 0
while currentLine:
    if currentLineNum % 1000000 == 0:
        print currentLineNum,"lines loaded."
    currentLineElements = currentLine.strip().split(" ")
    currentTerm = currentLineElements[0]
    currentTermListLength = int(currentLineElements[1])
    termANDUnprunedListLengthDict[currentTerm] = currentTermListLength 
    currentLine = inputFileHandler.readline()
    currentLineNum += 1
print "len(termANDUnprunedListLengthDict): ",len(termANDUnprunedListLengthDict)
inputFileHandler.close()

numOfQueriesHavingQID = 0
numOfQueriesHavingSearchContent = 0
numOfResultsForAllTheQueries = 0

outputFileHandlers = []
for i in range(0,11):
    outputFileName = "/home/vgc/wei/workspace/NYU_IRTK/results/20140716_attempt3_daatOR/gov2_head10KQueries_OR_daatOR_20140716_regressionTrainingFile_queryLength_" + str(i) + ".arff"
    outputFileHandler = open(outputFileName,"w")
    outputFileHandlers.append(outputFileHandler)

inputFileName1 = "/home/vgc/wei/workspace/NYU_IRTK/results/raw_results_AND_gov2_head10KQueries_20140715_from_vidaserver1"
inputFileHandler1 = open(inputFileName1,"r")
currentLine = inputFileHandler1.readline()
nextLine = ""
currentQID = ""

processFlag = True
outputLine = ""
while currentLine:
    # sampling parsing line:
    # qid: 701
    if currentLine.startswith("qid:"):
        currentQID = currentLine.strip().split(":")[1].strip()
        print
        print "QID:",currentQID
        print "QPC:",qidAndQPCForMaxScoreDict[currentQID]
        # qidAndQPCForMaxScoreDict[currentQID],"ms"
        outputLine = ""
        outputLine = str(currentQID) + ","
        # print "numOfQueriesHavingQID:",numOfQueriesHavingQID
        numOfQueriesHavingQID += 1
        # debug
        #if numOfQueriesHavingQID == 500:
        #    break

    # sample parsing line:
    # Search: u s  oil industry history
    if currentLine.startswith("Search:") and processFlag:
        elementList = currentLine.strip().split(" ")
        
        currentSearchContent = ""
        
        queryTermList = []
        for element in elementList[1:]:
            if element.strip() != "":
                queryTermList.append( element.strip() )
        
        for element in queryTermList:
            currentSearchContent += element + " "
        
        print "QueryContent:",currentSearchContent
        numOfQueriesHavingSearchContent += 1
        
        currentQueryTermIndexDict = {}
        nextLine = inputFileHandler1.readline()
        # print nextLine.strip()
        parsedFlag = True
        
        if nextLine.strip().endswith("is NOT in the lexicon."):
            parsedFlag = False
        
        if nextLine.strip() != "" and parsedFlag:
            nextLineElements = nextLine.strip().split(" ")
            # print "nextLineElements:",nextLineElements
            
            # sample parsing line: 
            # oil:0 industry:1 history:2 u:3 s:4
            for element in nextLineElements:
                if element.split(":")[0] not in currentQueryTermIndexDict:
                    currentQueryTermIndexDict[ int(element.split(":")[1]) ] = element.split(":")[0]
            
            # print "currentQueryTermIndexDict:",currentQueryTermIndexDict
            # print "len(currentQueryTermIndexDict):",len(currentQueryTermIndexDict)
            currentQueryLength = len(currentQueryTermIndexDict)
            print "QueryLength:",currentQueryLength
            if currentQueryLength not in queryLengthAndTotalCostDict:
                queryLengthAndTotalCostDict[currentQueryLength] = 0
            if currentQueryLength not in queryLengthAndNumOfQueriesBelongingToDict:
                queryLengthAndNumOfQueriesBelongingToDict[currentQueryLength] = 0
            
            queryLengthAndTotalCostDict[currentQueryLength] += int(qidAndQPCForMaxScoreDict[currentQID])
            queryLengthAndNumOfQueriesBelongingToDict[currentQueryLength] += 1
            QPCInTotal += int(qidAndQPCForMaxScoreDict[currentQID])
            
            if currentQueryLength > 1:
                for i in range(0,len(currentQueryTermIndexDict)):
                    currentTerm = currentQueryTermIndexDict[i]
                    print i,currentTerm,termANDUnprunedListLengthDict[currentTerm]
                    outputLine += str(termANDUnprunedListLengthDict[currentTerm]) + ","
                    if i == 0:
                        shortestList = int(termANDUnprunedListLengthDict[currentTerm])
                    if i == 1:
                        longerList = int(termANDUnprunedListLengthDict[currentTerm])
                costValue1 = shortestList * math.log(longerList/shortestList)
                costValue2 = math.log(longerList/shortestList)
            elif currentQueryLength == 1:
                currentTerm = currentQueryTermIndexDict[0]
                print i,currentTerm,termANDUnprunedListLengthDict[currentTerm]
                outputLine += str(termANDUnprunedListLengthDict[currentTerm]) + ","
                costValue1 = 0
                costValue2 = 0
            outputLine += str(costValue1) + ","
            outputLine += str(costValue2) + ","
            outputLine += qidAndQPCForMaxScoreDict[currentQID]
            
            # for queryLength = ALL file
            outputFileHandlers[0].write(outputLine + "\n")
            if currentQueryLength <= 10:
                outputFileHandlers[currentQueryLength].write(outputLine + "\n")
            
            
            # Now, it is time to read the actual training example line
            currentLine = inputFileHandler1.readline()
            numOfResultsForTheCurrentQuery = 0
        
    currentLine = inputFileHandler1.readline()

# print "numOfQueriesHavingQID:",numOfQueriesHavingQID
# print "numOfQueriesHavingSearchContent:",numOfQueriesHavingSearchContent
# print "numOfResultsForAllTheQueries:",numOfResultsForAllTheQueries
# print

print
print "OVERALL:"
tempCounter = 0
for currentQueryLength in range(1,11):
    avgQPCForThisQueryLength = queryLengthAndTotalCostDict[currentQueryLength] / queryLengthAndNumOfQueriesBelongingToDict[currentQueryLength]
    print currentQueryLength,queryLengthAndNumOfQueriesBelongingToDict[currentQueryLength],round(avgQPCForThisQueryLength,2),"ms"
    tempCounter += queryLengthAndNumOfQueriesBelongingToDict[currentQueryLength]

print "QPCInTotal:",QPCInTotal
print "tempCounter:",tempCounter
print "avg:",QPCInTotal/tempCounter
inputFileHandler1.close()
outputFileHandler.close()
print "Overall:"
print "output a set of files"